import { TextLoader } from "langchain/document_loaders/fs/text";
import path from "path";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";


export const load = async (dirname: string) => {
    const data_path = path.resolve(dirname, "..", "..", 'data', 'qiu.txt') 
    const loader = new TextLoader(data_path)
    const docs = await loader.load()
    return docs
}
export const split = async () => {
    const splitter = new RecursiveCharacterTextSplitter({
        chunkOverlap: 100,
        chunkSize: 500
    })
    return splitter
}